{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6af8ce8a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting ipywidgets\n",
      "  Downloading ipywidgets-8.1.7-py3-none-any.whl.metadata (2.4 kB)\n",
      "Requirement already satisfied: transformers in /opt/conda/lib/python3.11/site-packages (4.53.0)\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.11/site-packages (2.7.1)\n",
      "Requirement already satisfied: comm>=0.1.3 in /opt/conda/lib/python3.11/site-packages (from ipywidgets) (0.1.4)\n",
      "Requirement already satisfied: ipython>=6.1.0 in /opt/conda/lib/python3.11/site-packages (from ipywidgets) (8.16.1)\n",
      "Requirement already satisfied: traitlets>=4.3.1 in /opt/conda/lib/python3.11/site-packages (from ipywidgets) (5.11.2)\n",
      "Collecting widgetsnbextension~=4.0.14 (from ipywidgets)\n",
      "  Downloading widgetsnbextension-4.0.14-py3-none-any.whl.metadata (1.6 kB)\n",
      "Collecting jupyterlab_widgets~=3.0.15 (from ipywidgets)\n",
      "  Downloading jupyterlab_widgets-3.0.15-py3-none-any.whl.metadata (20 kB)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.11/site-packages (from transformers) (3.18.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in /opt/conda/lib/python3.11/site-packages (from transformers) (0.33.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.11/site-packages (from transformers) (2.3.1)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.11/site-packages (from transformers) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.11/site-packages (from transformers) (6.0.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.11/site-packages (from transformers) (2024.11.6)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.11/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.22,>=0.21 in /opt/conda/lib/python3.11/site-packages (from transformers) (0.21.2)\n",
      "Requirement already satisfied: safetensors>=0.4.3 in /opt/conda/lib/python3.11/site-packages (from transformers) (0.5.3)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.11/site-packages (from transformers) (4.66.1)\n",
      "Requirement already satisfied: typing-extensions>=4.10.0 in /opt/conda/lib/python3.11/site-packages (from torch) (4.14.0)\n",
      "Requirement already satisfied: sympy>=1.13.3 in /opt/conda/lib/python3.11/site-packages (from torch) (1.14.0)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.11/site-packages (from torch) (3.5)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.11/site-packages (from torch) (3.1.2)\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.11/site-packages (from torch) (2025.5.1)\n",
      "Requirement already satisfied: hf-xet<2.0.0,>=1.1.2 in /opt/conda/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.30.0->transformers) (1.1.5)\n",
      "Requirement already satisfied: backcall in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.2.0)\n",
      "Requirement already satisfied: decorator in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (5.1.1)\n",
      "Requirement already satisfied: jedi>=0.16 in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.19.1)\n",
      "Requirement already satisfied: matplotlib-inline in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.1.6)\n",
      "Requirement already satisfied: pickleshare in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.7.5)\n",
      "Requirement already satisfied: prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30 in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (3.0.39)\n",
      "Requirement already satisfied: pygments>=2.4.0 in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (2.16.1)\n",
      "Requirement already satisfied: stack-data in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (0.6.2)\n",
      "Requirement already satisfied: pexpect>4.3 in /opt/conda/lib/python3.11/site-packages (from ipython>=6.1.0->ipywidgets) (4.8.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /opt/conda/lib/python3.11/site-packages (from sympy>=1.13.3->torch) (1.3.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.11/site-packages (from jinja2->torch) (2.1.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests->transformers) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests->transformers) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests->transformers) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests->transformers) (2023.7.22)\n",
      "Requirement already satisfied: parso<0.9.0,>=0.8.3 in /opt/conda/lib/python3.11/site-packages (from jedi>=0.16->ipython>=6.1.0->ipywidgets) (0.8.3)\n",
      "Requirement already satisfied: ptyprocess>=0.5 in /opt/conda/lib/python3.11/site-packages (from pexpect>4.3->ipython>=6.1.0->ipywidgets) (0.7.0)\n",
      "Requirement already satisfied: wcwidth in /opt/conda/lib/python3.11/site-packages (from prompt-toolkit!=3.0.37,<3.1.0,>=3.0.30->ipython>=6.1.0->ipywidgets) (0.2.8)\n",
      "Requirement already satisfied: executing>=1.2.0 in /opt/conda/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (1.2.0)\n",
      "Requirement already satisfied: asttokens>=2.1.0 in /opt/conda/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (2.4.0)\n",
      "Requirement already satisfied: pure-eval in /opt/conda/lib/python3.11/site-packages (from stack-data->ipython>=6.1.0->ipywidgets) (0.2.2)\n",
      "Requirement already satisfied: six>=1.12.0 in /opt/conda/lib/python3.11/site-packages (from asttokens>=2.1.0->stack-data->ipython>=6.1.0->ipywidgets) (1.16.0)\n",
      "Downloading ipywidgets-8.1.7-py3-none-any.whl (139 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m139.8/139.8 kB\u001b[0m \u001b[31m1.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0mm\n",
      "\u001b[?25hDownloading jupyterlab_widgets-3.0.15-py3-none-any.whl (216 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m216.6/216.6 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0mta \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading widgetsnbextension-4.0.14-py3-none-any.whl (2.2 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.2/2.2 MB\u001b[0m \u001b[31m7.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0mm\n",
      "\u001b[?25hInstalling collected packages: widgetsnbextension, jupyterlab_widgets, ipywidgets\n",
      "Successfully installed ipywidgets-8.1.7 jupyterlab_widgets-3.0.15 widgetsnbextension-4.0.14\n"
     ]
    }
   ],
   "source": [
    "# Install required dependencies for Jupyter widgets, transformers and vector analysis\n",
    "!pip install ipywidgets transformers torch scikit-learn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5dd12514-e9a8-44b8-8a95-3aa3f622889e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "句向量维度: torch.Size([1, 768])\n"
     ]
    }
   ],
   "source": [
    "# 引入库\n",
    "from transformers import BertTokenizer, BertModel\n",
    "import torch\n",
    "\n",
    "# 加载模型\n",
    "tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')\n",
    "model = BertModel.from_pretrained('bert-base-chinese')\n",
    "\n",
    "# 处理文本\n",
    "text = \"深度学习改变世界\"\n",
    "inputs = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True)\n",
    "\n",
    "# 推理\n",
    "with torch.no_grad():\n",
    "    outputs = model(**inputs)\n",
    "    embeddings = outputs.last_hidden_state.mean(dim=1)  # 句向量\n",
    "\n",
    "print(\"句向量维度:\", embeddings.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8c1dfa48",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "============================================================\n",
      "Elasticsearch 向量搜索演示\n",
      "============================================================\n",
      "✅ Elasticsearch连接成功!\n",
      "集群名称: docker-cluster\n",
      "版本: 8.18.2\n",
      "✅ 向量索引 'text_vectors' 创建成功!\n"
     ]
    }
   ],
   "source": [
    "# Elasticsearch向量搜索演示\n",
    "import requests\n",
    "import json\n",
    "import numpy as np\n",
    "\n",
    "# ES连接配置\n",
    "es_host = \"http://host.docker.internal:9200\"\n",
    "\n",
    "def test_es_connection():\n",
    "    \"\"\"测试ES连接\"\"\"\n",
    "    try:\n",
    "        response = requests.get(es_host)\n",
    "        if response.status_code == 200:\n",
    "            cluster_info = response.json()\n",
    "            print(\"✅ Elasticsearch连接成功!\")\n",
    "            print(f\"集群名称: {cluster_info['cluster_name']}\")\n",
    "            print(f\"版本: {cluster_info['version']['number']}\")\n",
    "            return True\n",
    "        else:\n",
    "            print(\"❌ Elasticsearch连接失败!\")\n",
    "            return False\n",
    "    except Exception as e:\n",
    "        print(f\"❌ 连接错误: {e}\")\n",
    "        return False\n",
    "\n",
    "def create_vector_index():\n",
    "    \"\"\"创建支持向量搜索的索引\"\"\"\n",
    "    index_name = \"text_vectors\"\n",
    "    \n",
    "    # 删除已存在的索引\n",
    "    requests.delete(f\"{es_host}/{index_name}\")\n",
    "    \n",
    "    # 索引配置，包含向量字段\n",
    "    index_config = {\n",
    "        \"settings\": {\n",
    "            \"number_of_shards\": 1,\n",
    "            \"number_of_replicas\": 0\n",
    "        },\n",
    "        \"mappings\": {\n",
    "            \"properties\": {\n",
    "                \"text\": {\n",
    "                    \"type\": \"text\",\n",
    "                    \"analyzer\": \"ik_max_word\"\n",
    "                },\n",
    "                \"category\": {\n",
    "                    \"type\": \"keyword\"\n",
    "                },\n",
    "                \"vector\": {\n",
    "                    \"type\": \"dense_vector\",\n",
    "                    \"dims\": 768,  # BERT-base的向量维度\n",
    "                    \"index\": True,\n",
    "                    \"similarity\": \"cosine\"  # 使用余弦相似度\n",
    "                }\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "    \n",
    "    try:\n",
    "        response = requests.put(\n",
    "            f\"{es_host}/{index_name}\",\n",
    "            json=index_config,\n",
    "            headers={'Content-Type': 'application/json'}\n",
    "        )\n",
    "        \n",
    "        if response.status_code in [200, 201]:\n",
    "            print(f\"✅ 向量索引 '{index_name}' 创建成功!\")\n",
    "            return True\n",
    "        else:\n",
    "            print(f\"❌ 索引创建失败: {response.status_code}\")\n",
    "            print(response.text)\n",
    "            return False\n",
    "    except Exception as e:\n",
    "        print(f\"❌ 错误: {e}\")\n",
    "        return False\n",
    "\n",
    "# 测试连接并创建索引\n",
    "print(\"=\" * 60)\n",
    "print(\"Elasticsearch 向量搜索演示\")\n",
    "print(\"=\" * 60)\n",
    "\n",
    "if test_es_connection():\n",
    "    create_vector_index()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e40c4188",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "正在生成文档向量并插入到Elasticsearch...\n",
      "处理文档 1/10: 人工智能是计算机科学的一个分支，旨在创建能够模拟人类智能的机...\n",
      "处理文档 2/10: 机器学习是人工智能的子领域，专注于算法和统计模型...\n",
      "处理文档 3/10: 深度学习使用人工神经网络来模拟人脑的学习过程...\n",
      "处理文档 4/10: 自然语言处理帮助计算机理解和生成人类语言...\n",
      "处理文档 5/10: 计算机视觉使机器能够识别和理解图像内容...\n",
      "处理文档 6/10: 强化学习通过奖励和惩罚来训练智能体做出决策...\n",
      "处理文档 7/10: 神经网络是深度学习的基础，模拟大脑神经元的连接...\n",
      "处理文档 8/10: 监督学习使用标记数据来训练预测模型...\n",
      "处理文档 9/10: 无监督学习从未标记的数据中发现隐藏模式...\n",
      "处理文档 10/10: 大数据分析处理和分析大规模数据集以获取洞察...\n",
      "✅ 成功插入 10/10 个文档和向量\n",
      "索引已刷新，可以进行搜索\n"
     ]
    }
   ],
   "source": [
    "# 准备示例文档并生成向量\n",
    "def get_text_embedding(text):\n",
    "    \"\"\"获取文本的BERT向量\"\"\"\n",
    "    inputs = tokenizer(text, return_tensors=\"pt\", padding=True, truncation=True, max_length=512)\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        outputs = model(**inputs)\n",
    "        # 使用[CLS]标记的向量作为句子向量\n",
    "        embeddings = outputs.last_hidden_state[:, 0, :]  # 或者使用mean pooling\n",
    "        \n",
    "    return embeddings.numpy()[0]  # 转换为numpy数组\n",
    "\n",
    "def insert_documents_with_vectors():\n",
    "    \"\"\"插入文档和对应的向量到ES\"\"\"\n",
    "    index_name = \"text_vectors\"\n",
    "    \n",
    "    # 示例文档\n",
    "    documents = [\n",
    "        {\"text\": \"人工智能是计算机科学的一个分支，旨在创建能够模拟人类智能的机器\", \"category\": \"AI\"},\n",
    "        {\"text\": \"机器学习是人工智能的子领域，专注于算法和统计模型\", \"category\": \"ML\"},\n",
    "        {\"text\": \"深度学习使用人工神经网络来模拟人脑的学习过程\", \"category\": \"深度学习\"},\n",
    "        {\"text\": \"自然语言处理帮助计算机理解和生成人类语言\", \"category\": \"NLP\"},\n",
    "        {\"text\": \"计算机视觉使机器能够识别和理解图像内容\", \"category\": \"CV\"},\n",
    "        {\"text\": \"强化学习通过奖励和惩罚来训练智能体做出决策\", \"category\": \"强化学习\"},\n",
    "        {\"text\": \"神经网络是深度学习的基础，模拟大脑神经元的连接\", \"category\": \"神经网络\"},\n",
    "        {\"text\": \"监督学习使用标记数据来训练预测模型\", \"category\": \"监督学习\"},\n",
    "        {\"text\": \"无监督学习从未标记的数据中发现隐藏模式\", \"category\": \"无监督学习\"},\n",
    "        {\"text\": \"大数据分析处理和分析大规模数据集以获取洞察\", \"category\": \"大数据\"}\n",
    "    ]\n",
    "    \n",
    "    print(\"正在生成文档向量并插入到Elasticsearch...\")\n",
    "    success_count = 0\n",
    "    \n",
    "    for i, doc in enumerate(documents, 1):\n",
    "        try:\n",
    "            # 生成文本向量\n",
    "            print(f\"处理文档 {i}/{len(documents)}: {doc['text'][:30]}...\")\n",
    "            vector = get_text_embedding(doc['text'])\n",
    "            \n",
    "            # 准备文档数据\n",
    "            doc_data = {\n",
    "                \"text\": doc['text'],\n",
    "                \"category\": doc['category'],\n",
    "                \"vector\": vector.tolist()  # 转换为列表格式\n",
    "            }\n",
    "            \n",
    "            # 插入到ES\n",
    "            response = requests.post(\n",
    "                f\"{es_host}/{index_name}/_doc/{i}\",\n",
    "                json=doc_data,\n",
    "                headers={'Content-Type': 'application/json'}\n",
    "            )\n",
    "            \n",
    "            if response.status_code in [200, 201]:\n",
    "                success_count += 1\n",
    "            else:\n",
    "                print(f\"❌ 文档 {i} 插入失败: {response.status_code}\")\n",
    "                \n",
    "        except Exception as e:\n",
    "            print(f\"❌ 文档 {i} 处理错误: {e}\")\n",
    "    \n",
    "    print(f\"✅ 成功插入 {success_count}/{len(documents)} 个文档和向量\")\n",
    "    \n",
    "    # 刷新索引\n",
    "    requests.post(f\"{es_host}/{index_name}/_refresh\")\n",
    "    print(\"索引已刷新，可以进行搜索\")\n",
    "\n",
    "# 执行文档插入\n",
    "insert_documents_with_vectors()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4730c047",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "================================================================================\n",
      "向量搜索 vs 文本搜索对比演示\n",
      "================================================================================\n",
      "\n",
      "正在搜索: 智能机器人学习\n",
      "\n",
      "🔍 查询: '智能机器人学习'\n",
      "================================================================================\n",
      "\n",
      "🎯 向量搜索结果 (语义相似性):\n",
      "----------------------------------------\n",
      "1. [ML] 相似度: 0.924\n",
      "   内容: 机器学习是人工智能的子领域，专注于算法和统计模型\n",
      "\n",
      "2. [监督学习] 相似度: 0.923\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "3. [强化学习] 相似度: 0.922\n",
      "   内容: 强化学习通过奖励和惩罚来训练智能体做出决策\n",
      "\n",
      "\n",
      "📝 文本搜索结果 (关键词匹配):\n",
      "----------------------------------------\n",
      "1. [ML] 相关度: 2.801\n",
      "   内容: 机器学习是人工智能的子领域，专注于算法和统计模型\n",
      "\n",
      "2. [AI] 相关度: 2.340\n",
      "   内容: 人工智能是计算机科学的一个分支，旨在创建能够模拟人类智能的机器\n",
      "\n",
      "3. [强化学习] 相关度: 1.709\n",
      "   内容: 强化学习通过奖励和惩罚来训练智能体做出决策\n",
      "\n",
      "\n",
      "================================================================================\n",
      "\n",
      "正在搜索: 深度神经网络模型\n",
      "\n",
      "🔍 查询: '深度神经网络模型'\n",
      "================================================================================\n",
      "\n",
      "🎯 向量搜索结果 (语义相似性):\n",
      "----------------------------------------\n",
      "1. [监督学习] 相似度: 0.938\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "2. [深度学习] 相似度: 0.938\n",
      "   内容: 深度学习使用人工神经网络来模拟人脑的学习过程\n",
      "\n",
      "3. [神经网络] 相似度: 0.922\n",
      "   内容: 神经网络是深度学习的基础，模拟大脑神经元的连接\n",
      "\n",
      "\n",
      "📝 文本搜索结果 (关键词匹配):\n",
      "----------------------------------------\n",
      "1. [神经网络] 相关度: 6.294\n",
      "   内容: 神经网络是深度学习的基础，模拟大脑神经元的连接\n",
      "\n",
      "2. [深度学习] 相关度: 5.894\n",
      "   内容: 深度学习使用人工神经网络来模拟人脑的学习过程\n",
      "\n",
      "3. [监督学习] 相关度: 1.655\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "\n",
      "================================================================================\n",
      "\n",
      "正在搜索: 数据挖掘和分析\n",
      "\n",
      "🔍 查询: '数据挖掘和分析'\n",
      "================================================================================\n",
      "\n",
      "🎯 向量搜索结果 (语义相似性):\n",
      "----------------------------------------\n",
      "1. [大数据] 相似度: 0.941\n",
      "   内容: 大数据分析处理和分析大规模数据集以获取洞察\n",
      "\n",
      "2. [监督学习] 相似度: 0.929\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "3. [无监督学习] 相似度: 0.915\n",
      "   内容: 无监督学习从未标记的数据中发现隐藏模式\n",
      "\n",
      "\n",
      "📝 文本搜索结果 (关键词匹配):\n",
      "----------------------------------------\n",
      "1. [大数据] 相关度: 4.987\n",
      "   内容: 大数据分析处理和分析大规模数据集以获取洞察\n",
      "\n",
      "2. [监督学习] 相关度: 1.280\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "3. [无监督学习] 相关度: 1.241\n",
      "   内容: 无监督学习从未标记的数据中发现隐藏模式\n",
      "\n",
      "\n",
      "================================================================================\n",
      "\n",
      "正在搜索: 图像识别算法\n",
      "\n",
      "🔍 查询: '图像识别算法'\n",
      "================================================================================\n",
      "\n",
      "🎯 向量搜索结果 (语义相似性):\n",
      "----------------------------------------\n",
      "1. [监督学习] 相似度: 0.928\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "2. [无监督学习] 相似度: 0.928\n",
      "   内容: 无监督学习从未标记的数据中发现隐藏模式\n",
      "\n",
      "3. [深度学习] 相似度: 0.924\n",
      "   内容: 深度学习使用人工神经网络来模拟人脑的学习过程\n",
      "\n",
      "\n",
      "📝 文本搜索结果 (关键词匹配):\n",
      "----------------------------------------\n",
      "1. [CV] 相关度: 4.319\n",
      "   内容: 计算机视觉使机器能够识别和理解图像内容\n",
      "\n",
      "2. [ML] 相关度: 1.981\n",
      "   内容: 机器学习是人工智能的子领域，专注于算法和统计模型\n",
      "\n",
      "\n",
      "================================================================================\n",
      "\n",
      "正在搜索: 语言理解技术\n",
      "\n",
      "🔍 查询: '语言理解技术'\n",
      "================================================================================\n",
      "\n",
      "🎯 向量搜索结果 (语义相似性):\n",
      "----------------------------------------\n",
      "1. [监督学习] 相似度: 0.914\n",
      "   内容: 监督学习使用标记数据来训练预测模型\n",
      "\n",
      "2. [深度学习] 相似度: 0.910\n",
      "   内容: 深度学习使用人工神经网络来模拟人脑的学习过程\n",
      "\n",
      "3. [大数据] 相似度: 0.901\n",
      "   内容: 大数据分析处理和分析大规模数据集以获取洞察\n",
      "\n",
      "\n",
      "📝 文本搜索结果 (关键词匹配):\n",
      "----------------------------------------\n",
      "1. [NLP] 相关度: 4.113\n",
      "   内容: 自然语言处理帮助计算机理解和生成人类语言\n",
      "\n",
      "2. [CV] 相关度: 1.606\n",
      "   内容: 计算机视觉使机器能够识别和理解图像内容\n",
      "\n",
      "\n",
      "================================================================================\n"
     ]
    }
   ],
   "source": [
    "# 向量搜索功能实现\n",
    "def vector_search(query_text, k=5):\n",
    "    \"\"\"使用向量进行相似性搜索\"\"\"\n",
    "    index_name = \"text_vectors\"\n",
    "    \n",
    "    # 生成查询向量\n",
    "    query_vector = get_text_embedding(query_text)\n",
    "    \n",
    "    # 向量搜索查询\n",
    "    search_query = {\n",
    "        \"knn\": {\n",
    "            \"field\": \"vector\",\n",
    "            \"query_vector\": query_vector.tolist(),\n",
    "            \"k\": k,\n",
    "            \"num_candidates\": 100\n",
    "        },\n",
    "        \"_source\": [\"text\", \"category\"]\n",
    "    }\n",
    "    \n",
    "    try:\n",
    "        response = requests.post(\n",
    "            f\"{es_host}/{index_name}/_search\",\n",
    "            json=search_query,\n",
    "            headers={'Content-Type': 'application/json'}\n",
    "        )\n",
    "        \n",
    "        if response.status_code == 200:\n",
    "            return response.json()\n",
    "        else:\n",
    "            print(f\"❌ 向量搜索失败: {response.status_code}\")\n",
    "            return None\n",
    "    except Exception as e:\n",
    "        print(f\"❌ 向量搜索错误: {e}\")\n",
    "        return None\n",
    "\n",
    "def text_search(query_text, k=5):\n",
    "    \"\"\"传统文本搜索\"\"\"\n",
    "    index_name = \"text_vectors\"\n",
    "    \n",
    "    search_query = {\n",
    "        \"query\": {\n",
    "            \"match\": {\n",
    "                \"text\": query_text\n",
    "            }\n",
    "        },\n",
    "        \"size\": k,\n",
    "        \"_source\": [\"text\", \"category\"]\n",
    "    }\n",
    "    \n",
    "    try:\n",
    "        response = requests.post(\n",
    "            f\"{es_host}/{index_name}/_search\",\n",
    "            json=search_query,\n",
    "            headers={'Content-Type': 'application/json'}\n",
    "        )\n",
    "        \n",
    "        if response.status_code == 200:\n",
    "            return response.json()\n",
    "        else:\n",
    "            print(f\"❌ 文本搜索失败: {response.status_code}\")\n",
    "            return None\n",
    "    except Exception as e:\n",
    "        print(f\"❌ 文本搜索错误: {e}\")\n",
    "        return None\n",
    "\n",
    "def display_search_results(query, vector_results, text_results):\n",
    "    \"\"\"对比显示向量搜索和文本搜索结果\"\"\"\n",
    "    print(f\"\\n🔍 查询: '{query}'\")\n",
    "    print(\"=\" * 80)\n",
    "    \n",
    "    # 向量搜索结果\n",
    "    print(\"\\n🎯 向量搜索结果 (语义相似性):\")\n",
    "    print(\"-\" * 40)\n",
    "    if vector_results and vector_results.get('hits', {}).get('hits'):\n",
    "        for i, hit in enumerate(vector_results['hits']['hits'], 1):\n",
    "            score = hit['_score']\n",
    "            source = hit['_source']\n",
    "            print(f\"{i}. [{source['category']}] 相似度: {score:.3f}\")\n",
    "            print(f\"   内容: {source['text']}\")\n",
    "            print()\n",
    "    else:\n",
    "        print(\"❌ 没有找到向量搜索结果\")\n",
    "    \n",
    "    # 文本搜索结果\n",
    "    print(\"\\n📝 文本搜索结果 (关键词匹配):\")\n",
    "    print(\"-\" * 40)\n",
    "    if text_results and text_results.get('hits', {}).get('hits'):\n",
    "        for i, hit in enumerate(text_results['hits']['hits'], 1):\n",
    "            score = hit['_score']\n",
    "            source = hit['_source']\n",
    "            print(f\"{i}. [{source['category']}] 相关度: {score:.3f}\")\n",
    "            print(f\"   内容: {source['text']}\")\n",
    "            print()\n",
    "    else:\n",
    "        print(\"❌ 没有找到文本搜索结果\")\n",
    "\n",
    "# 测试不同的搜索查询\n",
    "test_queries = [\n",
    "    \"智能机器人学习\",\n",
    "    \"深度神经网络模型\",\n",
    "    \"数据挖掘和分析\",\n",
    "    \"图像识别算法\",\n",
    "    \"语言理解技术\"\n",
    "]\n",
    "\n",
    "print(\"\\n\" + \"=\" * 80)\n",
    "print(\"向量搜索 vs 文本搜索对比演示\")\n",
    "print(\"=\" * 80)\n",
    "\n",
    "for query in test_queries:\n",
    "    print(f\"\\n正在搜索: {query}\")\n",
    "    \n",
    "    # 执行向量搜索\n",
    "    vector_results = vector_search(query, k=3)\n",
    "    \n",
    "    # 执行文本搜索\n",
    "    text_results = text_search(query, k=3)\n",
    "    \n",
    "    # 显示对比结果\n",
    "    display_search_results(query, vector_results, text_results)\n",
    "    \n",
    "    print(\"\\n\" + \"=\"*80)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6d8a6157-dfa3-49be-8164-4545cbf65c09",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
